﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.Text.RegularExpressions;
namespace LinqAndFs
{
    class Program
    {
        static void Main(string[] args)
        {
            var rawpage = FetchUrlSimple(@"http://channel9.msdn.com/Events/MIX/MIX11");
            var page = Regex.Replace(rawpage, "&#?[a-z0-9]+;", " ");
            var titleMatches = Regex.Matches(page, "class=\"title\">(.*?)</a>");
            var abstractMatches = Regex.Matches(page.Replace("\n", " "), "class=\"description\">(.*?)</div>");
            var titles = titleMatches.Cast<Match>().Select(match => match.Groups[1].Value);
            var abstracts = abstractMatches.Cast<Match>().Select(match => match.Groups[1].Value);
            topKWords(titles, 20).ForEach(tuple=>
                {
                    Console.WriteLine(tuple.Item1 + " : " + tuple.Item2);
                }); ;
            Console.WriteLine("\n\n");
            topKWords(abstracts, 20).ForEach(tuple=>
                {
                    Console.WriteLine(tuple.Item1 + " : " + tuple.Item2);
                });

        }
        static string FetchUrlSimple(string url)
        {
            var req = WebRequest.Create(url);
            var response = req.GetResponse();
            using (var sr = new System.IO.StreamReader(response.GetResponseStream()))
            {
                return sr.ReadToEnd();
            }
        }
        static List<Tuple<string, int>> topKWords(IEnumerable<string> docs,int K)
        {
            char[] separator = { ' ', '\r', '\n', '-', '.', ',', '\t', '!', '?', '\'', ';', '/' };
            var stopwords = new HashSet<string>("a,able,about,across,after,all,almost,also,am,among,an,and,any,are,as,at,be,because,been,but,by,can,cannot,could,dear,did,do,does,either,else,ever,every,for,from,get,got,had,has,have,he,her,hers,him,his,how,however,i,if,in,into,is,it,its,just,least,let,like,likely,may,me,might,most,must,my,neither,no,nor,not,of,off,often,on,only,or,other,our,own,rather,said,say,says,she,should,since,so,some,than,that,the,their,them,then,there,these,they,this,tis,to,too,twas,us,wants,was,we,were,what,when,where,which,while,who,whom,why,will,with,would,yet,you,your".Split(','));
            return docs.SelectMany(doc => doc.Split(separator, StringSplitOptions.RemoveEmptyEntries)
                                .Select(word => word.ToLower())
                                .Where(word => !stopwords.Contains(word)))
                .GroupBy(a => a)
                .Select(wordgroup => Tuple.Create<string,int>(wordgroup.Key, wordgroup.Count()))
                .OrderByDescending(w2c => w2c.Item2)
                .Take(K)
                .ToList();
        }
        
    }
}
